import re

def tokenize_code(code):
    # 使用正则表达式进行分词，忽略空格，但是保留换行
    tokens = re.findall(r'[a-zA-Z_][a-zA-Z0-9_]*|\d+|\S|\n', code)
    return tokens

def py_code2word(path):
    with open(path, 'r', encoding='utf-8') as file:
        code_content = file.read()

    # 将读取的内容赋值给一个多行字符串变量
    code_snippet = """%s""" % code_content
    # 移除注释,还是说保留注释会更好呢？这里决定移除注释
    clean_code = re.sub('(?m)[ ]*(?:#.*)?$', '', code_snippet)
    tokens = tokenize_code(clean_code)
    return tokens

def go_Java_code2word(path):
    with open(path, 'r', encoding='utf-8', errors='ignore') as file:  # 忽略解码错误
        content = file.read()

    # 将读取的内容赋值给一个多行字符串变量
    go_code = """%s""" % content
    # 移除多行注释
    go_code = re.sub(r'(?m)[ ]*(?:/\*.*?\*/)', '', go_code, flags=re.DOTALL)
    # 移除单行注释
    go_code = re.sub(r'(?m)[ ]*(?://.*)?$', '', go_code)
    tokens = tokenize_code(go_code)
    return tokens

if __name__ == "__main__":
    file_path = 'hello.go'
    # 分词
    tokens = go_Java_code2word(file_path)
    print(tokens)
    print(len(tokens))